# -*- coding: utf-8 -*-

# Scrapy settings for xiangmu project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'xiangmu'

SPIDER_MODULES = ['xiangmu.spiders']
NEWSPIDER_MODULE = 'xiangmu.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'xiangmu (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
'Cookie':'PHPSESSID=79skfs5ppb95dfvu0i37v9mpi8; domain=hz; _ga=GA1.2.1446615438.1550564509; _gid=GA1.2.915016726.1550564509; yfx_c_g_u_id_10000001=_ck19021916215010707386732832855; Hm_lvt_94ed3d23572054a86ed341d64b267ec6=1550564511; _Jo0OQK=6DEADC4CA6F8606B8CE72B9F53CBD814DABA1E5E413ABD610249275018656F62224334C33E81091090D3A58B74C5FC50D7F244219A1E6F390DFCC50554B87763633FFBBE0C390CBD8D41F7A141E52F240311F7A141E52F24031425BE21A96A800C3800EF17EA71E8444GJ1Z1WQ==; zufang_BROWSES=32358952; yfx_f_l_v_t_10000001=f_t_1550564510060__r_t_1550564510060__v_t_1550570838547__r_c_0; _gat=1; Hm_lpvt_94ed3d23572054a86ed341d64b267ec6=1550571395',
}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'xiangmu.middlewares.XiangmuSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    'xiangmu.middlewares.XiangmuDownloaderMiddleware': 561,
   'xiangmu.middlewares.XiangmuSpiderMiddleware': 560,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   # 'xiangmu.pipelines.LeisuItem': 2,
   'xiangmu.pipelines.doubanImagesPipline': 1,
   'xiangmu.pipelines.MysqlPipeline': 300,

}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
IMAGES_STORE = 'download'

